<?php

// no direct access
defined('_JEXEC') or die('Restricted access');

require_once(JPATH_COMPONENT_SITE.DS.'helpers'.DS.'phpWebHacks.php');
require_once(JPATH_COMPONENT_SITE.DS.'helpers'.DS.'find_replace.php');
require_once(JPATH_COMPONENT_SITE.DS.'helpers'.DS.'resource_url.php');
require_once(JPATH_COMPONENT_SITE.DS.'helpers'.DS.'autocuttext.php');
require_once(JPATH_COMPONENT_SITE.DS.'helpers'.DS.'get_image.php');
require_once(JPATH_COMPONENT_SITE.DS.'helpers'.DS.'translator.php');

class FVB_Article extends JObject {
	//link configuration
	private $_link_id;
	
	//cms name
	private $_cms;
	
	//keywords to save article
	private $_keywords;
	
	//componnent configuration
	private $_global_configure;
	
	//link configuration
	private $_link_info;
	
	private $_arr_article;
	
	private $_browser;
	
	private $_arr_links = array();
	private $_arr_titles = array();
	private $_arr_intros = array();
	private $_arr_fulls = array();
	
	private $_link_page	=	"";
	private $_arr_detail_page	=	array();
	
	public function __construct()
	{
		$db	=& JFactory::getDBO();
		
		$this->_browser = new phpWebHacks();
		
		$config = &JComponentHelper::getParams('com_feed_manager');
		$this->_global_configure = $config;
		
		return true;
	}
	
	public function get_articles(){	
		return $this->_arr_article;
	}
	
	//copy all image from remote site then resize image (if you need)
	//public function will be called by job object
	public function do_fetch_images($max_widh = 0, $max_height = 0)
	{
		if (!$this->_arr_article) {
			return true;
		}
		
		foreach ($this->_arr_article as &$obj_article){			
			if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] fetch image BEGIN: ".$obj_article->link."");  }
			//process introtext
			$obj_article->intro_text = $this->_fetch_image_from_string($obj_article->intro_text, $obj_article->image_path, $max_widh, $max_height);
			//process fulltext
			$obj_article->full_text = $this->_fetch_image_from_string($obj_article->full_text, $obj_article->image_path, $max_widh, $max_height);
			if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] fetch image END.");  }
		}
		
		return true;
	}
	
	private function _fetch_image_from_string($str_in, $image_path, $max_widh, $max_height){
		//find all images
		if (!preg_match_all('/<img.*?src="([^"]+)"[^>]*>/i', $str_in, $matches)) {
			return $str_in;
		}
		
		$str_out = $str_in;
		foreach ($matches[1] as $str_img_src){
			if (JRequest::getInt('debug')) { FVB_utility::print_debug(4, "[ARTICLE] fetch single image BEGIN: ".$str_img_src);  }
			$Image_fetcher = new YOS_Get_Image($this->_link_id, $str_img_src, $image_path, $max_widh, $max_height);
			$str_file_stored = $Image_fetcher->get_image();
			if($str_file_stored === false){
				$arr_error = $Image_fetcher->getErrors();
				foreach ($arr_error as $str_error){
					$this->setError($str_error);
				}
				continue;
			}
			$file_name = JFile::getName($str_img_src);
			$new_file_name = JFile::getName($str_file_stored);
			$str_new_img_src = str_replace($file_name, $new_file_name, $str_img_src);
			$str_out = str_replace($str_img_src, $str_new_img_src, $str_out);
			if (JRequest::getInt('debug')) { FVB_utility::print_debug(4, "[ARTICLE] fetch single image END.");  }
		}
		
		return $str_out;
	}

	
	//try to rewrite article
	//public function will be called by job object
	public function do_rewrite()
	{
	}
	
	public function do_tidy_clean(){
		foreach ($this->_arr_article as &$obj_article){
			if (JRequest::getInt('debug')) { FVB_utility::print_debug(4, "[ARTICLE] Tidy clean an article BEGIN: ".$obj_article->link);  }
			$new_article = $this->_tidy_clean($obj_article);
			$obj_article = $new_article;
			if (JRequest::getInt('debug')) { FVB_utility::print_debug(4, "[ARTICLE] Tidy clean an article END.");  }
		}
		
		return true;
	}
	
	private function _tidy_clean($obj_article){
		$config = array(
			'indent' => TRUE,
			'output-xhtml' => TRUE,
			'show-body-only' => TRUE,
			'clean' => TRUE,
			'wrap' => 0
			);
		$tidy = new tidy();
		$obj_article->intro_text = $tidy->repairString($obj_article->intro_text, $config, 'utf8');
		$obj_article->full_text = $tidy->repairString($obj_article->full_text, $config, 'utf8');
		
		return $obj_article;
	}
		
	//public function will be called by job object
	public function do_get_articles()
	{
		/*********************************************************************
		 *************************GET LINKS & TITLES**************************
		 *********************************************************************/
		if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] Get links BEGIN");  }
		if($this->_get_links_titles() === false){
			$this->setError('[ARTICLE] Unable list of links and titles.');
			return false;
		}
		if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] Get links END");  }
		
		//no new articles
		if (!count($this->_arr_links)) {
			if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] No New articles");  }
			return true;
		}
		
		/*********************************************************************
		 *************************GET INTROTEXT*******************************
		 *********************************************************************/
		if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] Get introtext BEGIN");  }
		if ($this->_get_introtexts() === false) {
			$this->setError('[ARTICLE] Unable to get list of introtexts.');
			return false;
		}
		if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] Get introtext END");  }
		
		if (!count($this->_arr_links)) {
			//no new articles
			if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] No New articles #2");  }
			return true;
		}
		
		//remove empty introtext items
		$arr_new_links = array();
		$arr_new_titles = array();
		$arr_new_intros = array();
		for ($i = 0; $i < count($this->_arr_links); $i++){
			if (!$this->_arr_intros[$i]) {
				continue;
			}
			
			$arr_new_links[] = $this->_arr_links[$i];
			$arr_new_titles[] = $this->_arr_titles[$i];
			$arr_new_intros[] = $this->_arr_intros[$i];
		}
		$this->_arr_links = $arr_new_links;
		$this->_arr_titles = $arr_new_titles;
		$this->_arr_intros = $arr_new_intros;
		
		if (!count($this->_arr_links)) {
			if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] No New articles #3");  }
			return true;
		}
		
		/*********************************************************************
		 *************************GET FULLTEXT********************************
		 *********************************************************************/
		if(($this->_link_info->dataway == 2 || $this->_link_info->dataway == 3)){
			if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] Get fulltext from introtext (cut text) BEGIN");  }
			if($this->_get_fulltexts_from_introtexts() === false){
				$this->setError('[ARTICLE] Unable to get list of fulltexts from introtext.');
				return false;
			}
			if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] Get fulltext from introtext (cut text) END");  }
		}
		else {
			if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] Get fulltext BEGIN");  }
			if($this->_get_fulltexts() === false){
				$this->setError('[ARTICLE] Unable to get list of fulltexts.');
				return false;
			}
			if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] Get fulltext END");  }
		}
		
		/*********************************************************************
		 *************************GET MULTIPAGES******************************
		 *********************************************************************/
		if($this->_link_info->has_multiple_pages){
			if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] Get multiple pages BEGIN");  }
			if ($this->_get_multiple_page_data() === false) {
				$this->setError('[ARTICLE] Unable to get multiple page data');
				return false;
			}
			if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] Get multiple pages END");  }
		}
		
		//TODO: get meta keywords
		
		//TODO: get meta descriptions
		
		//final check count
		if (count($this->_arr_links) != count($this->_arr_intros) || count($this->_arr_links) != count($this->_arr_titles) || count($this->_arr_links) != count($this->_arr_fulls)) {
			$this->setError('[ARTICLE] counting elements has problem: ('.count($this->_arr_links).')links, ('.count($this->_arr_titles).')titles, ('.count($this->_arr_intros).')introtexts, ('.count($this->_arr_fulls).')fulltexts');
			return false;
		}
				
		$config =& JFactory::getConfig();
		for ($i = 0; $i < count($this->_arr_links); $i++){			
			$obj_article = new stdClass();		
			$tmp_path = $config->getValue('config.tmp_path').DS.'yos_news_crawler'.DS.$this->_link_id.DS.$i;
			$obj_article->image_path = $tmp_path;
			$obj_article->link = $this->_arr_links[$i];
			$obj_article->title = $this->_arr_titles[$i];
			$obj_article->intro_text = $this->_arr_intros[$i];
			$obj_article->full_text = $this->_arr_fulls[$i];
			
			$FVB_utility = new FVB_utility();
			$obj_article->title			= $FVB_utility->utf8_entity_decode($obj_article->title);
			$obj_article->intro_text	= $FVB_utility->utf8_entity_decode($obj_article->intro_text);
			$obj_article->full_text		= $FVB_utility->utf8_entity_decode($obj_article->full_text);
			
			//filter keyword to receive articles
			if ($this->_keywords) {
				$str_in = $obj_article->title.$obj_article->intro_text.$obj_article->full_text;
				if(!FVB_utility::test_keyword($str_in, $this->_keywords)){
					continue;
				}
			}
			
			$this->_arr_article[] = $obj_article;
		}
		
		if(count($this->_arr_article)){
			$this->_arr_article = array_reverse($this->_arr_article);
		}
		
		return true;
	}
	
	/**
	 * Get multiple page data
	 *
	 * @return boolean
	 */
	private function _get_multiple_page_data(){
		if (!$this->_link_info->pages_detect_reg) {
			$this->setError('[ARTICLE] Invalid pages detect regex.');
			return false;
		}
		
		for ($i = 0; $i < count($this->_arr_links); $i++){
			/*********************************************************************
			 *************************GET MULTIPAGE LINKS & DATA******************
			 *********************************************************************/
			if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] Get first page data BEGIN: ".$this->_arr_links[$i]."");  }
						
			if (!$this->_arr_fulls[$i]) {
				$this->setError('[ARTICLE] empty fulltext then go to the next round.');
				continue;
			}
			
			$arr_subpage_links = array();
			$arr_subpage_titles = array();
			$arr_subpage_datas = array();
			
			if (!key_exists($this->_arr_links[$i],$this->_arr_detail_page)) {
				$browser =& $this->_browser;
				$response = @$browser->get($this->_arr_links[$i]);
				if (!$response) {
					$this->setError('[ARTICLE] unable to connect: '.$this->_arr_links[$i]);
					continue;
				}
				$this->_arr_links[$i] = $browser->get_addressbar();
				$this->_arr_detail_page[$this->_arr_links[$i]]	= $response;
				
			}
			$response	=	$this->_arr_detail_page[$this->_arr_links[$i]];
			
			
			
			if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] Get first page data END.");  }
			
			if ($this->_link_info->multiple_way == 0) {
				//find all other pages
				if (!preg_match($this->_link_info->pages_detect_reg, $response, $matches)) {
					//NOT FOUND (THIS IS NOT AN ERROR)
					continue;
				}
				$str_pagenav = $matches[$this->_link_info->pages_detect_sub];
				if (!preg_match_all('/<a.*?href=(.*?)(\s|\/>|>)/ism', $str_pagenav, $matches)) {
					//NOT FOUND
					$error_counter++;
					continue;
				}
				
				$arr_tmp = $matches[1];
				for ($j = 0; $j < count($arr_tmp); $j++){
					$arr_tmp[$j] = preg_replace('/^("|\')/', '', $arr_tmp[$j]);
					$arr_tmp[$j] = preg_replace('/("|\')$/', '', $arr_tmp[$j]);
					if ($arr_tmp[$j] == $this->_arr_links[$i]) {
						continue;
					}
					$arr_subpage_links[] = $arr_tmp[$j];
				}
				
				$arr_subpage_links = array_unique($arr_subpage_links, SORT_STRING);

				for ($j = 0; $j < count($arr_subpage_links); $j++){
					$resource_url = new FVB_Resource_Url($arr_subpage_links[$j], $this->_arr_links[$i]);
					//Process link urls (href)
					$arr_subpage_links[$j] = $resource_url->process_single_url();
					
					if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] Get page data BEGIN: $arr_subpage_links[$j]");  }
					$obj_subpage = $this->_get_single_page_data($arr_subpage_links[$j]);
					if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] Get page data END.");  }
					$arr_subpage_titles[] = $obj_subpage->subtitle;
					$arr_subpage_datas[] = $obj_subpage->subpage;
				}
				
			}
			else {
				//find next page, after that go to the nextpage then do find next page again and again...
				$next_page_url = $this->_arr_links[$i];
				while (1) {
					if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] Get next page url BEGIN: $next_page_url");  }
					if(!$response = @$browser->get($next_page_url)){
						break;
					}
					$next_page_url = $browser->get_addressbar();
					
					//find all other pages
					if (!preg_match($this->_link_info->pages_detect_reg, $response, $matches)) {
						break;
					}
					
					$str_next_url = $matches[$this->_link_info->pages_detect_sub];
					
					$resource_url = new FVB_Resource_Url($str_next_url, $next_page_url);
					//Process link urls (href)
					$str_next_url = $resource_url->process_single_url();
					
					//to prevent loop
					if (in_array($str_next_url, $arr_subpage_links)) {
						break;
					}					
					if ($str_next_url == $this->_arr_links[$i]) {
						break;
					}
					
					if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] Get next page url END.");  }
					
					//get data
					if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] Get page data BEGIN: $str_next_url");  }
					$obj_subpage = $this->_get_single_page_data($str_next_url);
					if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] Get page data END.");  }
					
					$arr_subpage_titles[] = $obj_subpage->subtitle;
					$arr_subpage_datas[] = $obj_subpage->subpage;
									
					$arr_subpage_links[] = $str_next_url;						
					$next_page_url = $str_next_url;
				}
			}
			
			if (count($arr_subpage_links) != count($arr_subpage_datas)) {
				$this->setError('[ARTICLE] subpage links and subpage datas were not the same count: ('.count($arr_subpage_links).')subpage links and ('.count($arr_subpage_datas).')subpage datas');
				continue;
			}
			
			for ($j = 0; $j < count($arr_subpage_links);$j++){
				$this->_arr_fulls[$i] .= '<hr title="'.$arr_subpage_titles[$j].'" class="system-pagebreak" />'.$arr_subpage_datas[$j];
			}
			
		}
		
		return true;
	}
	
	/**
	 * Get fulltexts
	 *
	 * @return boolean
	 */
	private function _get_fulltexts(){
		$FVB_utility = new FVB_utility();
		
		$arr_fulls = array();
		
		$error_counter = 0;
		//connect to the detail pages then get intro
		foreach ($this->_arr_links as &$link){
			if (!key_exists($link,$this->_arr_detail_page)) {
				$browser =& $this->_browser;
				$response = @$browser->get($link);
				
				if (!$response) {
					$arr_fulls[] = '';
					$error_counter++;
					continue;
				}	
				$charset = $browser->get_charset();
				$this->_arr_detail_page[$link]=$response;
			}
			
			$response	=	$this->_arr_detail_page[$link];
			
			
			if (!preg_match($this->_link_info->getfull_reg, $response, $matches)) {
				$arr_fulls[] = '';
				$error_counter++;
				continue;
			}						
			$str_fulltext = $matches[$this->_link_info->getfull_sub];
			
			$str_fulltext = $FVB_utility->charset_process($str_fulltext, $charset);
			
			$arr_fulls[] = $str_fulltext;
		}
		
		$percentage = $error_counter/count($this->_arr_links)*100;
		if ($percentage > intval($this->_global_configure->get('percentage'))) {
			$this->setError('[ARTICLE] Too many errors when get fulltexts: '.$percentage.'%');
			return false;
		}
		
		if (count($this->_arr_links) != count($arr_fulls)) {
			$this->setError('[ARTICLE] links and fulltexts were not the same count: ('.count($this->_arr_links).')links and ('.count($arr_fulls).')fulltexts');
			return false;
		}

		//process links and image source
		for ($i = 0; $i < count($arr_fulls); $i++){
			$resource_url = new FVB_Resource_Url($arr_fulls[$i], $this->_arr_links[$i]);
			//Process link urls (href)
			$arr_fulls[$i] = $resource_url->process_href();
			
			//Process image source urls (src)
			$arr_fulls[$i] = $resource_url->process_src();
			
		}
		
		//Find and replace here 
		if($this->_link_info->find_replace_full){
			for ($i = 0; $i < count($arr_fulls);$i++){
				$arr_fulls[$i] = YOS_Find_Replace::find_replace($arr_fulls[$i], explode("\n", $this->_link_info->find_replace_full));
			}
		}
		
		$this->_arr_fulls = $arr_fulls;
		
		return true;
	}
	
	/**
	 * get fulltexts from introtexts by using cuttext
	 *
	 * @return boolean
	 */
	private function _get_fulltexts_from_introtexts(){
		$arr_fulls = array();
		
		for ($i = 0; $i < count($this->_arr_intros); $i++){
			if($this->_link_info->cut_content && $this->_link_info->numberword > 0){
				//Cut text here
				$auto_cutter = new AutoCutText($this->_arr_intros[$i], $this->_link_info->numberword);
				$str_intro = $auto_cutter->getIntro();
				if ($str_intro) {
					$this->_arr_intros[$i] = $str_intro;
					$arr_fulls[$i] = $auto_cutter->getFulltext();
				}
				else {
					$arr_fulls[$i] = '';
				}
			}
			else {
				$arr_fulls[$i] = '';
			}
		}
		$this->_arr_fulls = $arr_fulls;
		
		return true;
	}
	
	/**
	 * get list of introtexts
	 *
	 * @return boolean
	 */
	private function _get_introtexts(){
		$FVB_utility = new FVB_utility();
		$db	=& JFactory::getDBO();
		
		$arr_intros = array();
		
		if($this->_link_info->dataway == 0 || $this->_link_info->dataway == 3){
			$str_real_url = $this->_link_info->link;
			
			$charset = false;
			//connect to the link page then get intro
			if ($this->_link_info->category == 1) {
				//xml
				$options['rssUrl']		= $this->_link_info->link;
				$options['cache_time']	= 60;
				$rssDoc =& JFactory::getXMLparser('RSS', $options);
				
				if (!$rssDoc) {
					$this->setError('[ARTICLE] Unable to read xml file: '.$this->_link_info->link);
					return false;
				}
				
				$items	=	$rssDoc->get_items();
				$items	=	array_slice($items , 0, count($this->_arr_links));
				
				foreach ($items as $item) {					
					$arr_intros[] = $item->get_description();
				}
			}
			else {
				//http
				
				if (!$this->_link_page) {
					
					$browser =& $this->_browser;					
					$response = @$browser->get($this->_link_info->link);
					if (!$response) {
						$this->setError('[ARTICLE] unable to connect: '.$this->_link_info->link);
						return false;
					}
					$this->_link_page	=	$response;
					$str_real_url = $browser->get_addressbar();
					$charset = $browser->get_charset();
				}
				
				$response	=	$this->_link_page;
			
				if (!preg_match_all($this->_link_info->getintro_reg, $response, $matches)) {
					$this->setError('[ARTICLE] invalid intro regex: '.$this->_link_info->getintro_reg);
					return false;
				}
				
				$arr_intros = array_slice($matches[$this->_link_info->getintro_sub], 0, count($this->_arr_links));
			}
			
			if (count($this->_arr_links) != count($arr_intros)) {
				$this->setError('[ARTICLE] links and intros were not the same count: ('.count($this->_arr_links).')links and ('.count($arr_intros).')intros');
				return false;
			}
			
			for ($i = 0; $i < count($arr_intros); $i++){
				$resource_url = new FVB_Resource_Url($arr_intros[$i], $str_real_url);
				//Process link urls (href)
				$arr_intros[$i] = $resource_url->process_href();
				
				//Process image source urls (src)
				$arr_intros[$i] = $resource_url->process_src();
				
				$arr_intros[$i]	= $FVB_utility->charset_process($arr_intros[$i], $charset);
			}
			
			//long time no connect, try to check mysql connection;
			$FVB_utility->mysql_check_connection();
			
			//remove fetched links
			$arr_links_tmp = array();
			$arr_titles_tmp = array();
			$arr_intros_tmp = array();
			for ($i = 0; $i < count($this->_arr_links); $i++){
				$query = "SELECT COUNT(*) FROM #__yos_content_link WHERE `link` = ".$db->Quote($this->_arr_links[$i])." 
					AND `cms` = ".$db->Quote($this->_cms);
				$db->setQuery($query);
				$is_exist = $db->loadResult();
				if ($db->getErrorNum()) {
					$arr_error = debug_backtrace();
					$this->setError('[ARTICLE] MySQL error: '.$db->getErrorMsg()."\n FILE: ".$arr_error[0]['file']."\n LINE: ".$arr_error[0]['line']);
					return false;
				}
				if ($is_exist) {
					continue;
				}
				$arr_links_tmp[] = $this->_arr_links[$i];
				$arr_titles_tmp[] = $this->_arr_titles[$i];
				$arr_intros_tmp[] = $arr_intros[$i];
			}
			$this->_arr_links = $arr_links_tmp;
			$this->_arr_titles = $arr_titles_tmp;
			$arr_intros = $arr_intros_tmp;
		}
		else {
			//long time no connect, try to check mysql connection;
			$FVB_utility->mysql_check_connection();
			
			//remove fetched links
			$arr_links_tmp = array();
			$arr_titles_tmp = array();
			for ($i = 0; $i < count($this->_arr_links); $i++){
				$query = "SELECT COUNT(*) FROM #__yos_content_link WHERE `link` = ".$db->Quote($this->_arr_links[$i])." 
					AND `cms` = ".$db->Quote($this->_cms);
				$db->setQuery($query);
				$is_exist = $db->loadResult();
				if ($db->getErrorNum()) {
					$arr_error = debug_backtrace();
					$this->setError('[ARTICLE] MySQL error: '.$db->getErrorMsg()."\n FILE: ".$arr_error[0]['file']."\n LINE: ".$arr_error[0]['line']);
					return false;
				}
				if ($is_exist) {
					continue;
				}
				$arr_links_tmp[] = $this->_arr_links[$i];
				$arr_titles_tmp[] = $this->_arr_titles[$i];
			}
			$this->_arr_links = $arr_links_tmp;
			$this->_arr_titles = $arr_titles_tmp;
			
			if (!count($this->_arr_links)) {
				//do not need to fetch content anymore
				return true;
			}
			
			//connect to the detail pages then get intro
			$error_counter = 0;
			foreach ($this->_arr_links as &$link){
				
				if (!key_exists($link,$this->_arr_detail_page)) {
					$browser =& $this->_browser;
					$response = @$browser->get($link);
					if (!$response) {
						$error_counter++;
						$arr_intros[] = '';
						continue;
					}
					$link = $browser->get_addressbar();
					$charset = $browser->get_charset();
					$this->_arr_detail_page[$link]	=	$response;					
				}
				$response	=	$this->_arr_detail_page[$link];
				
				if (!preg_match($this->_link_info->getintro_reg, $response, $matches)) {
					$error_counter++;
					$arr_intros[] = '';
					continue;
				}
				$str_intro = $matches[$this->_link_info->getintro_sub];
				$str_intro = $FVB_utility->charset_process($str_intro, $charset);
				
				$arr_intros[] = $str_intro;
			}
			
			$percentage = $error_counter/count($this->_arr_links)*100;
			if ($percentage > intval($this->_global_configure->get('percentage'))) {
				$this->setError('[ARTICLE] Too many errors when get introtexts: '.$percentage.'%');
				return false;
			}
			
			if (count($this->_arr_links) != count($arr_intros)) {
				$this->setError('[ARTICLE] links and intros were not the same count: ('.count($this->_arr_links).')links and ('.count($arr_intros).')intros');
				return false;
			}
						
			for ($i = 0; $i < count($arr_intros); $i++){
				$resource_url = new FVB_Resource_Url($arr_intros[$i], $this->_arr_links[$i]);
				//Process link urls (href)
				$arr_intros[$i] = $resource_url->process_href();
				
				//Process image source urls (src)
				$arr_intros[$i] = $resource_url->process_src();
			}
		}
		
		//Find and replace here 
		if($this->_link_info->find_replace_intro){
			for ($i = 0; $i < count($arr_intros);$i++){
				$arr_intros[$i] = YOS_Find_Replace::find_replace($arr_intros[$i], explode("\n", $this->_link_info->find_replace_intro));
			}
		}
		
		$this->_arr_intros = $arr_intros;
		
		return true;
	}
	
	/**
	 * Get list of links and titles
	 *
	 * @return boolean, true if success else return false.
	 */
	private function _get_links_titles(){
		$FVB_utility = new FVB_utility();
		$db	=& JFactory::getDBO();
								
		$arr_links = array();
		$arr_titles = array();
		
		$charset = false;
		if ($this->_link_info->category == 1) {
			//xml
			$options['rssUrl']		= $this->_link_info->link;
			$options['cache_time']	= 60;
			$rssDoc =& JFactory::getXMLparser('RSS', $options);
			
			if (!$rssDoc) {
				$this->setError('[ARTICLE] Unable to read xml file: '.$this->_link_info->link);
				return false;
			}
			
			$items	=	$rssDoc->get_items();

			foreach ($items as $item) {
				$arr_links[] = $item->get_link();
				$arr_titles[] = $item->get_title();
			}
		}
		else {
			//http
			if (!$this->_link_page) {
				$browser =& $this->_browser;
				$response = @$browser->get($this->_link_info->link);
				
				if (!$response) {
					$this->setError('[ARTICLE] unable to connect: '.$this->_link_info->link);
					return false;
				}	
				$this->_link_page	=	$response;
				$charset = $browser->get_charset();
			}
			
			$response	=	$this->_link_page;
			
			if (!preg_match_all($this->_link_info->getlink_reg, $response, $matches)) {
				$this->setError('[ARTICLE] Invalid link regex: '.$this->_link_info->getlink_reg);
				return false;
			}
			$arr_links = $matches[$this->_link_info->getlink_sub];
			
			if (!preg_match_all($this->_link_info->gettitle_reg, $response, $matches)) {
				$this->setError('[ARTICLE] Invalid title regex: '.$this->_link_info->gettitle_reg);
				return false;
			}
			$arr_titles = $matches[$this->_link_info->gettitle_sub];
		}
		
		$arr_links	=	array_slice($arr_links , 0, $this->_link_info->numberStore);
		$arr_titles	=	array_slice($arr_titles , 0, $this->_link_info->numberStore);
		
		if (count($arr_links) != count($arr_titles)) {
			$this->setError('[ARTICLE] links and titles were not the same count: ('.count($arr_links).')links and ('.count($arr_titles).')titles');
			return false;
		}
		
		$arr_links_tmp = array();
		$arr_titles_tmp = array();
		$error_counter = 0;
		for ($i = 0; $i < count($arr_links); $i++){
			$resource_url = new FVB_Resource_Url($arr_links[$i], $this->_link_info->link);
			//Process link urls (href)
			$str_link = $resource_url->process_single_url();
			
			//tring to connect then get the real links (pass 302 redirects)
			$browser =& $this->_browser;
			$response = @$browser->head($str_link);
			if (!$response) {
				$error_counter++;
				continue;
			}
			$str_real_link = $browser->get_addressbar();
			$arr_links_tmp[] = $str_real_link;
			$arr_titles_tmp[] = $FVB_utility->charset_process($arr_titles[$i], $charset);
		}
		$arr_links = $arr_links_tmp;
		$arr_titles = $arr_titles_tmp;
		
		//find and replace links here
		if($this->_link_info->find_replace_links){
			for ($i = 0; $i < count($arr_links);$i++){
				$arr_links[$i] = YOS_Find_Replace::find_replace($arr_links[$i], explode("\n", $this->_link_info->find_replace_links));
			}
		}
		
		//find and replace titles
		if($this->_link_info->find_replace_titles){
			for ($i = 0; $i < count($arr_titles);$i++){
				$arr_titles[$i] = YOS_Find_Replace::find_replace($arr_titles[$i], explode("\n", $this->_link_info->find_replace_titles));
			}
		}
		
		$this->_arr_links = $arr_links;
		$this->_arr_titles = $arr_titles;
		
		return true;
	}
	
	/**
	 * Get single page data from list of multiple page
	 *
	 * @param string $page_url
	 * @return object with subtitle and subpage string properties
	 */
	private function _get_single_page_data($page_url){
		$FVB_utility = new FVB_utility();
		
		$obj_return = new stdClass();
		$obj_return->subtitle = '';
		$obj_return->subpage = '';
		
		$sub_link =	$page_url;
		
		$browser =& $this->_browser;
		$response = @$browser->get($sub_link);
		
		$str_subtitle	= "";
		$str_subpage 	= "";
		if(!$response){
			if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] cannot get sub page data: $sub_link");  }
			return $obj_return;
		}
		$sub_link = $browser->get_addressbar();
		$charset = $browser->get_charset();
		
		//find all other page title
		if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] find sub page title");  }
		if ($this->_link_info->multipage_title_reg && preg_match($this->_link_info->multipage_title_reg, $response, $matches)) {
			$str_subtitle=$matches[$this->_link_info->multipage_title_sub];
			
			$str_subtitle = $FVB_utility->charset_process($str_subtitle, $charset);
		}
		
		if (JRequest::getInt('debug')) { FVB_utility::print_debug(3, "[ARTICLE] find sub page body");  }
		if (preg_match($this->_link_info->multipage_reg, $response, $matches)){
			$str_subpage = $matches[$this->_link_info->multipage_sub];
			
			$str_subpage = $FVB_utility->charset_process($str_subpage, $charset);
		}
		
		$resource_url = new FVB_Resource_Url($str_subpage, $sub_link);
		//Process link urls (href)
		$str_subpage = $resource_url->process_href();
		
		//Process image source urls (src)
		$str_subpage = $resource_url->process_src();
		
		$str_subpage = YOS_Find_Replace::find_replace($str_subpage, explode("\n", $this->_link_info->find_replace_multipage));
		
		$obj_return = new stdClass();
		$obj_return->subtitle = $str_subtitle;
		$obj_return->subpage = $str_subpage;
		return $obj_return;
	}
}
